#include "../../../include/offline/DictProducer.hpp"
#include "../../../include/offline/SplitTool.hpp"
#include "../../../include/offline/Configuration.hpp"
#include <sys/types.h>
#include <dirent.h>
#include <string.h>
#include <fstream>
#include <sstream>
#include <iostream>

using std::cout;
using std::endl;
using std::ifstream;
using std::istringstream;
using std::ofstream;


DictProducer::DictProducer(const string &dir)
	: _endir(dir) {
	string enDictPath = Configuration::getInstance()->getConfigMap()["enDict"];
	string enDictIndex = Configuration::getInstance()->getConfigMap()["enDictIndex"];
	string enStopDictPath = Configuration::getInstance()->getConfigMap()["enStop"];

	loadStopWord(enStopDictPath);

	buildEnDict();
	storeDict(enDictPath.c_str());
	buildIndex();
	storeIndex(enDictIndex.c_str());
	cout << "En Dict and Index was built" << endl;
} 

DictProducer::DictProducer(const string &dir, SplitTool *splitTool)
	: _splitTool(splitTool), _cndir(dir) {
	string cnDictPath = Configuration::getInstance()->getConfigMap()["cnDict"];
	string cnDictIndex = Configuration::getInstance()->getConfigMap()["cnDictIndex"];
	string cnStopDictPath = Configuration::getInstance()->getConfigMap()["cnStop"];

	loadStopWord(cnStopDictPath);

	buildCnDict();
	storeDict(cnDictPath.c_str());
	buildIndex();
	storeIndex(cnDictIndex.c_str());
	cout << "Cn Dict and Index was built" << endl;
} 

void DictProducer::loadStopWord(string stopDictPath) {
	_stopWord.clear();
	ifstream ifs(stopDictPath);
	if (!ifs.good()) {
		cout << "DictProducer::loadStopWord: ifs is not good" << endl;
		return;
	}
	string line, word;
	while (getline(ifs, line)) {
		istringstream iss(line);
		while (iss >> word) {
			_stopWord.insert(word);
		}
	}
	ifs.close();
}

void DictProducer::buildEnDict() {
	stringstream ss;
	string line, word, in_word;

	_files.clear();
	showFiles();
	getFiles(_endir);

	for (auto &file : _files) {
		ifstream ifs(file);
		if (!ifs.good()) {
			cout << "DictProducer::build: ifstream open error" << endl;
			return;
		}

		while (getline(ifs, line)) {
			in_word.clear();
			istringstream iss(line);

			while (iss >> word) {
				in_word.clear();
				for (char c : word) {
					//处理大写
					if (c >= 'A' && c <= 'Z') {
						c += 32;
					}
					//处理非字母字符
					if (isalpha(c)) {
						in_word += c;
					} else
						break;
				}
                //遇到连字符
				if (in_word[in_word.size() - 1] == '-')
					in_word.resize(in_word.size() - 1);
                //不在stopword集合里面
				if (in_word != " " && find(_stopWord.begin(), _stopWord.end(), in_word) == _stopWord.end())
					++_dict[in_word];
				else
					continue;
			}
		}
		ifs.close();
	}
}


//该部分较难
void DictProducer::buildCnDict() {
	getFiles(_cndir);
	
    for (auto &file : _files) {
		ifstream ifs(file, std::ios::ate);
		if (!ifs.good()) {
			cout << "DictProducer::buildCnDict: ifs is not good" << endl;
			return;
		}

		size_t length = ifs.tellg(); 
		ifs.seekg(std::ios_base::beg);
		char *buff = new char[length + 1]; 
		ifs.read(buff, length + 1);
		string txt(buff);

		delete[] buff;

		vector<string> tmp = _splitTool->cut(txt);

		for (auto &elem : tmp) {
			if (!_stopWord.count(elem) && getByteNum_UTF8(elem[0]) == 3) {
				auto exist = _dict.find(elem);
				if (exist != _dict.end()) {
					++_dict[elem];
				} else {
                    _dict[elem] = 1;
				}
			}
		}
		ifs.close();
	}
}

//构建索引
//借助函数getByteNum_UTF8()每次可以获取一个字符占几个字节
//从而实现每次取一个字符，构建中英文索引均适用
void DictProducer::buildIndex() {
	int i = 0;  //i简单理解为控制下标或行号
	for (auto &elem : _dict) {
		string word = elem.first;
		size_t charNums = word.size() / getByteNum_UTF8(word[0]); //获取字符长度
		//buildIndex扫描过程没有对比stopWords(对比stopword太耗时)
        for (size_t idx = 0, n = 0; n != charNums; ++idx, ++n) {
			size_t charLen = getByteNum_UTF8(word[idx]);
			string subWord = word.substr(idx, charLen);
			_index[subWord].insert(i);
			idx += (charLen - 1);
		}
		++i;
	}
}

void DictProducer::storeDict(const char *filepath) {
	ofstream ofs(filepath);
	if (!ofs.good()) {
		cout << "DictProduc::storeDict: error in storeDict" << endl;
		return;
	}
	for (auto &it : _dict) {
		ofs << it.first << "  " << it.second << endl;
	}
	ofs.close();
}

void DictProducer::storeIndex(const char *filepath) {
	ofstream ofs(filepath);
	if (!ofs.good()) {
		cout << "DictProducer::storeIndex: ofs is not good" << endl;
		return;
	}
	for (auto &it : _index) {
		ofs << it.first << "  ";        //保存key
		for (auto &set_it : it.second) {//保存value(set<int>)
			ofs << set_it << " ";
		}
		ofs << endl;
	}
	ofs.close();
}

//显示读取到的文件
void DictProducer::showFiles() const {
	for (auto &elem : _files) {
		cout << elem << endl;
	}
}

//显示读取到的字典
void DictProducer::showDict() const {
	for (auto &it : _index) {
		cout << it.first << " ";
		for (auto &set_it : it.second) {
			cout << set_it << " ";
		}
	}
}

//获取指定目录下的文件列表
void DictProducer::getFiles(string dir) { 

	DIR *dirp = opendir(dir.c_str());
	if (!dirp) {
		cout << "DictProducer::getFiles: open dir error" << endl;
	}

	struct dirent *pdirent;
	while ((pdirent = readdir(dirp)) != NULL) {
		if (strcmp(pdirent->d_name, ".") == 0 || strcmp(pdirent->d_name, "..") == 0)
			continue;

		string filePath;
		filePath = dir + "/" + pdirent->d_name;

		_files.push_back(filePath);
	}
	closedir(dirp);
}

//获取指定的一个字符所占的字节数
size_t DictProducer::getByteNum_UTF8(const char byte) {
	int byteNum = 0;
	for (size_t i = 0; i < 6; ++i) {
		if (byte & (1 << (7 - i)))
			++byteNum;
		else
			break;
	}
	return byteNum == 0 ? 1 : byteNum;
}
